
import json 
import numpy as np
import pandas as pd
import re, nltk, spacy, string
import en_core_web_sm
nlp = en_core_web_sm.load()

#Importing libraries required for the case study and to plot charts
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import os
from plotly.offline import plot
import plotly.graph_objects as go
import plotly.express as px
%matplotlib inline

#Importing Regular Expressions for String manipulation
import re

# hide warnings
import warnings
warnings.filterwarnings('ignore')

# set options
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from pprint import pprint


# root path for kaggle 
root_path = 'complaints-2021-05-14_08_16.json'


# Opening JSON file 
f = open(root_path) 

# returns JSON object as a dictionary 
data = json.load(f)


df=pd.json_normalize(data)
df_to_visualize = pd.json_normalize(data)


# Inspect the dataframe to understand the given data.
df.head(10)


#print the column names
df.columns

Index(['_index', '_type', '_id', '_score', '_source.tags', '_source.zip_code',
       '_source.complaint_id', '_source.issue', '_source.date_received',
       '_source.state', '_source.consumer_disputed', '_source.product',
       '_source.company_response', '_source.company', '_source.submitted_via',
       '_source.date_sent_to_company', '_source.company_public_response',
       '_source.sub_product', '_source.timely',
       '_source.complaint_what_happened', '_source.sub_issue',
       '_source.consumer_consent_provided'],
      dtype='object')


#Assign new column names
df = df[['_source.complaint_what_happened','_source.product','_source.sub_product',]]
df.tail()


# lets rename the column names to be more read friendly
# we have retained product and sub_product only for verification purpose ,
# these columns will not be used in the Topic modelling
df = df.rename(columns={'_source.complaint_what_happened': 'complaint_text', '_source.product': 'category','_source.sub_product': 'sub_category'})
df.head()


# lets merge the category and sub-category this will help us in deciding the topics after NMF modelling
df['category'] = df['category'] + '+' + df['sub_category']
df = df.drop(['sub_category'],axis= 1)
df.head()


# lets check the nan complaints
df.complaint_text.isnull().sum()

0


# lets check the empty string complaints
len(df[df['complaint_text']==''])

57241


#Assign nan in place of blanks in the complaints column
# lets replace empty complain with nan value
df[df['complaint_text']==''] = np.nan
df.complaint_text.isnull().sum()

57241


#Remove all rows where complaints column is nan
# Lets drop all the rows where complaint_text is null
df = df[~df['complaint_text'].isnull()]
df.complaint_text.isnull().sum()

0


# Preparing text for Topic modelling
def clean_text(text):
    text = text.lower()  # Make the text lowercase
    text = re.sub('\[.*\]','', text).strip() # Remove text in square brackets
    text = text.translate(str.maketrans('', '', string.punctuation)) # Remove punctuation
    text = re.sub('\S*\d\S*\s*','', text).strip()  # Remove words containing numbers
    return text.strip()


df.complaint_text = df.complaint_text.apply(lambda x: clean_text(x))
df.complaint_text.head()

1     good morning my name is xxxx xxxx and i apprec...
2     i upgraded my xxxx xxxx card in and was told b...
10    chase card was reported on however fraudulent ...
11    on while trying to book a xxxx  xxxx  ticket i...
14    my grand son give me check for i deposit it in...
Name: complaint_text, dtype: object


#Function to Lemmatize the texts
stopwords = nlp.Defaults.stop_words
def lemmatizer(text):
    doc = nlp(text)
    sent = [token.lemma_ for token in doc if not token.text in set(stopwords)]
    return ' '.join(sent)


# Lemmatize the complaints
df['lemma'] =  df.complaint_text.apply(lambda x: lemmatizer(x))
df.head()


#Created a dataframe('df_clean') that will have only the complaints and the lemmatized complaints 
df_clean = df[['complaint_text','lemma','category']]
df_clean.head()


#function to extract the POS tags NN
def extract_pos_tags(text):
    doc = nlp(text)
    sent = [token.text for token in doc if token.tag_ == 'NN']
    return ' '.join(sent)


#The clean dataframe should now contain the raw complaint, lemmatized complaint and the complaint after removing POS tags.
df_clean['complaint_POS_removed'] =  df_clean.lemma.apply(lambda x: extract_pos_tags(x))
df_clean.head()


# visualizing the data according to the 'Complaint' character length

plt.figure(figsize=(10,6))
doc_lens = [len(d) for d in df_clean.complaint_POS_removed]
plt.hist(doc_lens, bins = 50)

(array([9.235e+03, 6.357e+03, 2.724e+03, 1.277e+03, 7.580e+02, 2.740e+02,
        1.500e+02, 7.100e+01, 6.300e+01, 3.900e+01, 3.700e+01, 1.300e+01,
        1.100e+01, 1.100e+01, 8.000e+00, 8.000e+00, 7.000e+00, 2.000e+00,
        3.000e+00, 3.000e+00, 7.000e+00, 2.000e+00, 2.000e+00, 0.000e+00,
        0.000e+00, 0.000e+00, 1.000e+00, 0.000e+00, 0.000e+00, 2.000e+00,
        0.000e+00, 0.000e+00, 0.000e+00, 4.000e+00, 0.000e+00, 0.000e+00,
        0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 1.000e+00, 0.000e+00,
        0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 1.000e+00,
        0.000e+00, 1.000e+00]),
 array([    0. ,   248.5,   497. ,   745.5,   994. ,  1242.5,  1491. ,
         1739.5,  1988. ,  2236.5,  2485. ,  2733.5,  2982. ,  3230.5,
         3479. ,  3727.5,  3976. ,  4224.5,  4473. ,  4721.5,  4970. ,
         5218.5,  5467. ,  5715.5,  5964. ,  6212.5,  6461. ,  6709.5,
         6958. ,  7206.5,  7455. ,  7703.5,  7952. ,  8200.5,  8449. ,
         8697.5,  8946. ,  9194.5,  9443. ,  9691.5,  9940. , 10188.5,
        10437. , 10685.5, 10934. , 11182.5, 11431. , 11679.5, 11928. ,
        12176.5, 12425. ]),
 <BarContainer object of 50 artists>)


df_to_visualize.rename(columns=lambda x: x.replace('_source.', ''), inplace=True)


df_to_visualize.head()


import matplotlib.pyplot as plt
import plotly.graph_objects as go

product_counts = df_to_visualize['product'].value_counts()

fig = go.Figure(data=go.Bar(x=product_counts.index, y=product_counts))

fig.update_layout(
    title='Complaints by Product Categories',
    xaxis_title='Product Category',
    yaxis_title='Number of Complaints',
    xaxis_tickangle=90,  # Rotate x-axis labels for better visibility
    width=800,
    height=1000
)

fig.show()


import matplotlib.pyplot as plt
import plotly.graph_objects as go

response_counts = df_to_visualize['company_response'].value_counts()

fig = go.Figure(data=go.Pie(labels=response_counts.index, values=response_counts))

fig.update_layout(
    title='Complaint Resolution Status',
    yaxis_title='',
    width=800,
    height=500
)

fig.show()


import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import plotly.graph_objects as go

df_to_visualize['date_received'] = pd.to_datetime(df_to_visualize['date_received'])

fig = go.Figure()

fig.add_trace(go.Histogram(
    x=df_to_visualize['date_received'],
    nbinsx=20,
    marker=dict(color='blue', opacity=0.7),
    opacity=0.7
))

fig.update_layout(
    title='Distribution of Complaints Over Time',
    xaxis_title='Date Received',
    yaxis_title='Number of Complaints',
    xaxis_tickangle=-45,  # Rotate x-axis labels for better visibility
    width=800,
    height=500
)

fig.show()


state_counts = df_to_visualize['state'].value_counts().reset_index()
state_counts.columns = ['State', 'Number of Complaints']

top_10_states = state_counts.nlargest(10, 'Number of Complaints')

fig = px.bar(
    top_10_states,
    x='Number of Complaints',
    y='State',
    title='Top 10 States with the Highest Number of Complaints',
    labels={'Number of Complaints': 'Number of Complaints'},
    orientation='h',
    width=800,
    height=500
)

fig.update_layout(
    xaxis_title='Number of Complaints',
)

fig.show()


plt.figure(figsize=(6, 4))
sns.countplot(data=df_to_visualize, x='timely')
plt.title('Timely vs. Untimely Complaints')
plt.xlabel('Timely Complaint')
plt.ylabel('Number of Complaints')
plt.show()


from wordcloud import WordCloud

text_data = ' '.join(df_to_visualize['complaint_what_happened'].dropna())
wordcloud = WordCloud(width=800, height=400, background_color='white').generate(text_data)

plt.figure(figsize=(10, 6))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis('off')
plt.title('Word Cloud of Complaint Text')
plt.show()


wordcloud = WordCloud(stopwords=stopwords,max_words=40, background_color='white').generate(str(df_clean['complaint_POS_removed']))

print(wordcloud)
plt.figure(figsize=(10,6))
plt.imshow(wordcloud)
plt.axis('off')
plt.show()

<wordcloud.wordcloud.WordCloud object at 0x0000024D9E09D7C8>


#Removing -PRON- from the text corpus
df_clean['Complaint_clean'] = df_clean['complaint_POS_removed'].str.replace('-PRON-', '')
df_clean = df_clean.drop(['complaint_POS_removed'],axis = 1)


# to find the top 30 unigram,bigram,trigram frequency 
# among the complaints in the cleaned datafram(df_clean).

def get_top_n_bigram(text, ngram=1, top=None):
    vec = CountVectorizer(ngram_range=(ngram, ngram), stop_words='english').fit(text)
    bag_of_words = vec.transform(text)

    sum_words = bag_of_words.sum(axis=0) 
    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]

    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
    return words_freq[:top]


top_30_unigrams = get_top_n_bigram(df_clean.Complaint_clean,ngram=1, top=30)
top_30_bigrams = get_top_n_bigram(df_clean.Complaint_clean,ngram=2, top=30)
top_30_trigrams = get_top_n_bigram(df_clean.Complaint_clean,ngram=3, top=30)


#Print the top 10 words in the unigram frequency
print('Top 10 unigrams:\n')
top_30_unigrams[:10]

Top 10 unigrams:

[('chase', 53415),
 ('account', 40962),
 ('credit', 33554),
 ('xxxx', 31273),
 ('card', 29835),
 ('payment', 21064),
 ('bank', 17034),
 ('time', 15871),
 ('charge', 15292),
 ('day', 12831)]


#Plot graph for the top 30 words in the unigram frequency
df1 = pd.DataFrame(top_30_unigrams, columns = ['unigram' , 'count'])
plt.figure(figsize=(12,6))
fig = sns.barplot(x=df1['unigram'], y=df1['count'])
plt.xticks(rotation = 80)
plt.show()


#Print the top 10 words in the bigram frequency
print('Top 10 bigrams:\n')
top_30_bigrams[:10]

Top 10 bigrams:

[('credit card', 12773),
 ('credit report', 3893),
 ('chase credit', 3070),
 ('customer service', 2923),
 ('account chase', 2432),
 ('chase bank', 2119),
 ('check account', 2113),
 ('chase account', 2034),
 ('xxxx xxxx', 1955),
 ('contact chase', 1893)]


#Plot graph for the top 30 words in the bigram frequency
df2 = pd.DataFrame(top_30_bigrams, columns = ['bigram' , 'count'])
plt.figure(figsize=(12,6))
fig = sns.barplot(x=df2['bigram'], y=df2['count'])
plt.xticks(rotation = 80)
plt.show()


#Print the top 10 words in the trigram frequency
print('Top 10 trigrams:\n')
top_30_trigrams[:10]

Top 10 trigrams:

[('chase credit card', 2196),
 ('credit card account', 1050),
 ('credit card company', 978),
 ('credit card chase', 738),
 ('xxxx credit card', 652),
 ('chase customer service', 559),
 ('inquiry credit report', 483),
 ('credit card credit', 438),
 ('charge credit card', 406),
 ('credit card xxxx', 385)]


#Plot graph for the top 30 words in the trigram frequency
df3 = pd.DataFrame(top_30_trigrams, columns = ['trigram' , 'count'])
plt.figure(figsize=(12,6))
fig = sns.barplot(x=df3['trigram'], y=df3['count'])
plt.xticks(rotation = 80)
plt.show()


df_clean['Complaint_clean'] = df_clean['Complaint_clean'].str.replace('xxxx','')


#All masked texts has been removed
df_clean.head()


#to initialise the TfidfVectorizer 
tfidf = TfidfVectorizer(min_df=2, max_df=0.95, stop_words='english')


#to create the Document Term Matrix by transforming the complaints column present in df_clean.
dtm = tfidf.fit_transform(df_clean.Complaint_clean)


tfidf.get_feature_names_out()[:10]

array(['aarp', 'abandonment', 'abatement', 'abbreviation', 'abd',
       'abhorrent', 'abide', 'ability', 'abovementione', 'abovereference'],
      dtype=object)


len(tfidf.get_feature_names_out())

6483


from sklearn.decomposition import NMF


#Load your nmf_model with the n_components i.e 5
num_topics =  5 

#keep the random_state =40
nmf_model = NMF(n_components=num_topics, random_state=40)

W1 = nmf_model.fit_transform(dtm)
H1 = nmf_model.components_


#nmf_model.fit(dtm)
#len(tfidf.get_feature_names())


#Print the Top15 words for each of the topics
num_words=15

vocab = np.array(tfidf.get_feature_names_out())

top_words = lambda t: [vocab[i] for i in np.argsort(t)[:-num_words-1:-1]]
topic_words = ([top_words(t) for t in H1])
topics = [' '.join(t) for t in topic_words]


vocab

array(['aarp', 'abandonment', 'abatement', ..., 'zombie', 'zone', 'zoom'],
      dtype=object)


topics

['account check money bank deposit chase fund day branch transfer number business transaction customer claim',
 'card credit chase balance account score limit year purchase point month application service reason time',
 'payment loan mortgage chase month modification time home pay year rate balance property statement date',
 'report inquiry credit information company reporting debt account file identity remove dispute theft score letter',
 'charge dispute fee chase transaction merchant claim purchase refund fraud service email time contact statement']


#Created the best topic for each complaint in terms of integer value 0,1,2,3 & 4

colnames = ["Topic" + str(i) for i in range(nmf_model.n_components)]
docnames = ["Doc" + str(i) for i in range(len(df_clean.Complaint_clean))]
df_doc_topic = pd.DataFrame(np.round(W1, 2), columns=colnames, index=docnames)
significant_topic = np.argmax(df_doc_topic.values, axis=1)
df_doc_topic['dominant_topic'] = significant_topic
df_doc_topic.head()


#Assign the best topic to each of the cmplaints in Topic Column
df_clean['Topic'] = significant_topic


pd.set_option('display.max_colwidth', -1)


df_clean[['complaint_text','Complaint_clean','category','Topic']][df_clean.Topic==4].head(30)


# Print the first 10 Complaint for each of the Topics
temp =df_clean[['complaint_text','Complaint_clean','category','Topic']].groupby('Topic').head(10)
temp.sort_values('Topic')


#Create the dictionary of Topic names and Topics
topic_mapping = {
    0: 'Bank Account services',
    1: 'Credit card or prepaid card',
    2: 'Others',
    3: 'Theft/Dispute Reporting',
    4: 'Mortgage/Loan'
}

#Replace Topics with Topic Names
df_clean['Topic'] = df_clean['Topic'].map(topic_mapping)


df_clean.head()


# Plot graph to check class imbalance
plt.figure(figsize=(12,6))
sns.countplot(x='Topic',data=df_clean)

<AxesSubplot:xlabel='Topic', ylabel='count'>


#Keep the columns"complaint_what_happened" & "Topic" only in the new dataframe --> training_data

training_data = df_clean[['complaint_text','Topic']]


training_data.head()


## Reverse topic names mapping for supervised learning

reverse_topic_mapping = {
    'Bank Account services' :0,
    'Credit card or prepaid card':1,
    'Others':2,
    'Theft/Dispute Reporting':3,
    'Mortgage/Loan':4
}
#Replace Topics with Topic Names
training_data['Topic'] = training_data['Topic'].map(reverse_topic_mapping)
training_data.head()


training_data[['complaint_text','Topic']][training_data.Topic==2].head(30)


# x - y  split
X = training_data.complaint_text
y = training_data.Topic


#Write your code to get the Vector count 

# Fit transform the X
count_vect = CountVectorizer()
X_vect = count_vect.fit_transform(X)


from sklearn.feature_extraction.text import TfidfTransformer
#Write your code here to transform the word vector to tf-idf
#Fit transform word vector to TF-IDF
tfidf_transformer = TfidfTransformer()
X_tfidf = tfidf_transformer.fit_transform(X_vect)


from sklearn.model_selection import train_test_split
# train test split
X_train, X_test, y_train, y_test = train_test_split(X_tfidf, y, test_size=0.25, random_state=40, stratify=y)


# Write your code here to build any 3 models and evaluate them using the required metrics

from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier


from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV

from sklearn import metrics
from sklearn.metrics import roc_auc_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_recall_fscore_support

from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import ConfusionMatrixDisplay


# utlity function to display the model report

def display_classification_report(model,metric):
    
    y_train_pred_proba = model.predict_proba(X_train)
    y_test_pred_proba = model.predict_proba(X_test)
    roc_auc_score_train = round(roc_auc_score(y_train, y_train_pred_proba,average='weighted',multi_class='ovr'),2)
    roc_auc_score_test = round(roc_auc_score(y_test, y_test_pred_proba,average='weighted',multi_class='ovr'),2)
    print("ROC AUC Score Train:", roc_auc_score_train)
    print("ROC AUC Score Test:", roc_auc_score_test)
    metric.append(roc_auc_score_train)
    metric.append(roc_auc_score_test)

    y_train_pred = model.predict(X_train)
    y_test_pred = model.predict(X_test)
  
    precision_train,recall_train,fscore_train,support_train=precision_recall_fscore_support(y_train,y_train_pred,average='weighted')
    precision_test,recall_test,fscore_test,support_test=precision_recall_fscore_support(y_test,y_test_pred,average='weighted')
    
    acc_score_train = round(accuracy_score(y_train,y_train_pred),2)
    acc_score_test = round(accuracy_score(y_test,y_test_pred),2)
    
    metric.append(acc_score_train)
    metric.append(acc_score_test)
    metric.append(round(precision_train,2))
    metric.append(round(precision_test,2))
    metric.append(round(recall_train,2))
    metric.append(round(recall_test,2))
    metric.append(round(fscore_train,2))
    metric.append(round(fscore_test,2))
    
    print('Train Accuracy :',acc_score_train)
    print('Test Accuracy :',acc_score_test)
        
    model_report_train = classification_report(y_train,y_train_pred)
    model_report_test = classification_report(y_test,y_test_pred)
    
    print('Classification Report for Train:\n',model_report_train)
    print('Classification Report for Test:\n',model_report_test)


    # Plot the confusion matrix
    fig, ax = plt.subplots(figsize=(12, 8))
    # Create the confusion matrix
    cm = confusion_matrix(y_test, y_test_pred)
    cmp = ConfusionMatrixDisplay(cm, display_labels=model.classes_)
    cmp.plot(ax=ax)
    plt.xticks(rotation=80)

    plt.show();


# create 5 folds
folds = StratifiedKFold(n_splits = 5, shuffle = True, random_state = 40)


# utility method to get the GridSearchCV object
def grid_search(model,folds,params,scoring):
    
    grid_search = GridSearchCV(model,
                                cv=folds, 
                                param_grid=params, 
                                scoring=scoring, 
                                n_jobs=-1, verbose=1)
    return grid_search


# utility function to display the best score and parameters of the GridSearchCV model
def print_best_score_params(model):
    print("Best Score: ", model.best_score_)
    print("Best Hyperparameters: ", model.best_params_)


# Multinomial Naive Bayes Classification

# create MNB model object
mnb = MultinomialNB()
# fit model
mnb.fit(X_train, y_train)
# display classification report
metric1=[]
display_classification_report(mnb,metric1)

ROC AUC Score Train: 0.95
ROC AUC Score Test: 0.93
Train Accuracy : 0.68
Test Accuracy : 0.64
Classification Report for Train:
               precision    recall  f1-score   support

           0       0.55      0.96      0.70      4361
           1       0.81      0.50      0.61      3109
           2       0.75      0.88      0.81      3795
           3       0.98      0.20      0.34      1627
           4       0.88      0.45      0.60      2912

    accuracy                           0.68     15804
   macro avg       0.79      0.60      0.61     15804
weighted avg       0.76      0.68      0.65     15804

Classification Report for Test:
               precision    recall  f1-score   support

           0       0.52      0.95      0.67      1454
           1       0.76      0.40      0.53      1037
           2       0.71      0.85      0.77      1265
           3       0.93      0.17      0.28       542
           4       0.88      0.41      0.56       970

    accuracy                           0.64      5268
   macro avg       0.76      0.56      0.56      5268
weighted avg       0.72      0.64      0.61      5268


# Logistic Regression Classification
log_reg = LogisticRegression(random_state=40,solver='liblinear')
# fit model
log_reg.fit(X_train,y_train)
# display classification report
metric2=[]
display_classification_report(log_reg,metric2)

ROC AUC Score Train: 1.0
ROC AUC Score Test: 0.99
Train Accuracy : 0.94
Test Accuracy : 0.9
Classification Report for Train:
               precision    recall  f1-score   support

           0       0.93      0.97      0.95      4361
           1       0.95      0.94      0.94      3109
           2       0.95      0.98      0.96      3795
           3       0.95      0.88      0.91      1627
           4       0.95      0.91      0.93      2912

    accuracy                           0.94     15804
   macro avg       0.95      0.93      0.94     15804
weighted avg       0.94      0.94      0.94     15804

Classification Report for Test:
               precision    recall  f1-score   support

           0       0.89      0.93      0.91      1454
           1       0.90      0.89      0.89      1037
           2       0.91      0.95      0.93      1265
           3       0.93      0.82      0.87       542
           4       0.91      0.86      0.88       970

    accuracy                           0.90      5268
   macro avg       0.91      0.89      0.90      5268
weighted avg       0.90      0.90      0.90      5268


# Decision Tree Classification
dtc = DecisionTreeClassifier(random_state=40)
# fit model
dtc.fit(X_train,y_train)
# Decision Tree Classification Report
metric3=[]
display_classification_report(dtc,metric3)

ROC AUC Score Train: 1.0
ROC AUC Score Test: 0.85
Train Accuracy : 1.0
Test Accuracy : 0.76
Classification Report for Train:
               precision    recall  f1-score   support

           0       1.00      1.00      1.00      4361
           1       1.00      1.00      1.00      3109
           2       1.00      1.00      1.00      3795
           3       1.00      1.00      1.00      1627
           4       1.00      1.00      1.00      2912

    accuracy                           1.00     15804
   macro avg       1.00      1.00      1.00     15804
weighted avg       1.00      1.00      1.00     15804

Classification Report for Test:
               precision    recall  f1-score   support

           0       0.77      0.78      0.77      1454
           1       0.77      0.77      0.77      1037
           2       0.82      0.81      0.81      1265
           3       0.74      0.71      0.72       542
           4       0.68      0.70      0.69       970

    accuracy                           0.76      5268
   macro avg       0.76      0.75      0.75      5268
weighted avg       0.76      0.76      0.76      5268


# Random Forest Classification

rf = RandomForestClassifier(n_estimators = 500,random_state=40, n_jobs = -1,oob_score=True)
# fit model
rf.fit(X_train,y_train)

# oob score
print('OOB SCORE :',rf.oob_score_)

# Random Forest Classification Report
metric4=[]
display_classification_report(rf,metric4)

OOB SCORE : 0.821311060491015
ROC AUC Score Train: 1.0
ROC AUC Score Test: 0.98
Train Accuracy : 1.0
Test Accuracy : 0.83
Classification Report for Train:
               precision    recall  f1-score   support

           0       1.00      1.00      1.00      4361
           1       1.00      1.00      1.00      3109
           2       1.00      1.00      1.00      3795
           3       1.00      1.00      1.00      1627
           4       1.00      1.00      1.00      2912

    accuracy                           1.00     15804
   macro avg       1.00      1.00      1.00     15804
weighted avg       1.00      1.00      1.00     15804

Classification Report for Test:
               precision    recall  f1-score   support

           0       0.78      0.93      0.85      1454
           1       0.83      0.80      0.81      1037
           2       0.85      0.90      0.87      1265
           3       0.91      0.68      0.78       542
           4       0.86      0.71      0.78       970

    accuracy                           0.83      5268
   macro avg       0.85      0.80      0.82      5268
weighted avg       0.84      0.83      0.83      5268


# Multinomial Naive Bayes with GridSearchCV

mnb = MultinomialNB()

mnb_params = {  
'alpha': (1, 0.1, 0.01, 0.001, 0.0001)  
}

# create gridsearch object
grid_search_mnb = grid_search(mnb, folds, mnb_params, scoring=None)

# fit model
grid_search_mnb.fit(X_train, y_train)

# print best hyperparameters
print_best_score_params(grid_search_mnb)

# Random Forest Classification Report
metric5=[]
display_classification_report(grid_search_mnb,metric5)

Fitting 5 folds for each of 5 candidates, totalling 25 fits
Best Score:  0.7574663722023555
Best Hyperparameters:  {'alpha': 0.01}
ROC AUC Score Train: 0.99
ROC AUC Score Test: 0.94
Train Accuracy : 0.89
Test Accuracy : 0.76
Classification Report for Train:
               precision    recall  f1-score   support

           0       0.88      0.93      0.90      4361
           1       0.89      0.83      0.86      3109
           2       0.91      0.92      0.92      3795
           3       0.87      0.85      0.86      1627
           4       0.90      0.87      0.88      2912

    accuracy                           0.89     15804
   macro avg       0.89      0.88      0.88     15804
weighted avg       0.89      0.89      0.89     15804

Classification Report for Test:
               precision    recall  f1-score   support

           0       0.76      0.84      0.80      1454
           1       0.70      0.64      0.67      1037
           2       0.79      0.85      0.82      1265
           3       0.78      0.67      0.72       542
           4       0.75      0.69      0.72       970

    accuracy                           0.76      5268
   macro avg       0.76      0.74      0.74      5268
weighted avg       0.76      0.76      0.75      5268


# Logistic Regression with GridSearchCV

# logistic regression
log_reg = LogisticRegression()

# hyperparameter for Logistic Regression
log_params = {'C': [0.01, 1, 10], 
          'penalty': ['l1', 'l2'],
          'solver': ['liblinear','newton-cg','saga']
         }

# create gridsearch object
grid_search_log = grid_search(log_reg, folds, log_params, scoring=None)

# fit model
grid_search_log.fit(X_train, y_train)

# print best hyperparameters
print_best_score_params(grid_search_log)

# Random Forest Classification Report
metric6=[]
display_classification_report(grid_search_log,metric6)

Fitting 5 folds for each of 18 candidates, totalling 90 fits
Best Score:  0.9233733516472515
Best Hyperparameters:  {'C': 1, 'penalty': 'l1', 'solver': 'saga'}
ROC AUC Score Train: 1.0
ROC AUC Score Test: 0.99
Train Accuracy : 0.94
Test Accuracy : 0.93
Classification Report for Train:
               precision    recall  f1-score   support

           0       0.93      0.95      0.94      4361
           1       0.94      0.94      0.94      3109
           2       0.96      0.96      0.96      3795
           3       0.93      0.90      0.91      1627
           4       0.93      0.91      0.92      2912

    accuracy                           0.94     15804
   macro avg       0.94      0.93      0.93     15804
weighted avg       0.94      0.94      0.94     15804

Classification Report for Test:
               precision    recall  f1-score   support

           0       0.94      0.94      0.94      1454
           1       0.92      0.92      0.92      1037
           2       0.94      0.96      0.95      1265
           3       0.92      0.89      0.90       542
           4       0.92      0.92      0.92       970

    accuracy                           0.93      5268
   macro avg       0.93      0.92      0.93      5268
weighted avg       0.93      0.93      0.93      5268


# Decision Tree Classification wuth GridSearchCV

dtc = DecisionTreeClassifier(random_state=40)

dtc_params = {
    'max_depth': [5,10,20,30],
    'min_samples_leaf': [5,10,20,30]
}

# create gridsearch object
grid_search_dtc = grid_search(dtc, folds, dtc_params, scoring='roc_auc_ovr')

# fit model
grid_search_dtc.fit(X_train, y_train)

# print best hyperparameters
print_best_score_params(grid_search_dtc)

# Random Forest Classification Report
metric7=[]
display_classification_report(grid_search_dtc,metric7)

Fitting 5 folds for each of 16 candidates, totalling 80 fits
Best Score:  0.9302492043101971
Best Hyperparameters:  {'max_depth': 30, 'min_samples_leaf': 30}
ROC AUC Score Train: 0.97
ROC AUC Score Test: 0.94
Train Accuracy : 0.82
Test Accuracy : 0.78
Classification Report for Train:
               precision    recall  f1-score   support

           0       0.83      0.83      0.83      4361
           1       0.80      0.86      0.83      3109
           2       0.86      0.88      0.87      3795
           3       0.77      0.74      0.75      1627
           4       0.80      0.73      0.77      2912

    accuracy                           0.82     15804
   macro avg       0.81      0.81      0.81     15804
weighted avg       0.82      0.82      0.82     15804

Classification Report for Test:
               precision    recall  f1-score   support

           0       0.79      0.80      0.79      1454
           1       0.77      0.83      0.80      1037
           2       0.82      0.83      0.82      1265
           3       0.72      0.65      0.68       542
           4       0.75      0.69      0.72       970

    accuracy                           0.78      5268
   macro avg       0.77      0.76      0.76      5268
weighted avg       0.78      0.78      0.78      5268


# Random Forest Classifier with GridSearchCV

rfc = RandomForestClassifier(random_state=40, n_jobs = -1,oob_score=True)

# hyperparameters for Random Forest
rfc_params = {'max_depth': [10,20,30,40],
          'min_samples_leaf': [5,10,15,20,30],
          'n_estimators': [100,200,500,700]
        }

# create gridsearch object
grid_search_rfc = grid_search(rfc, folds, rfc_params, scoring='roc_auc_ovr')

# fit model
grid_search_rfc.fit(X_train, y_train)

# oob score
print('OOB SCORE :',grid_search_rfc.best_estimator_.oob_score_)

# print best hyperparameters
print_best_score_params(grid_search_rfc)

# Random Forest Classification Report
metric8=[]
display_classification_report(grid_search_rfc,metric8)

Fitting 5 folds for each of 80 candidates, totalling 400 fits
OOB SCORE : 0.803594026828651
Best Score:  0.9712320811544322
Best Hyperparameters:  {'max_depth': 40, 'min_samples_leaf': 5, 'n_estimators': 700}
ROC AUC Score Train: 0.99
ROC AUC Score Test: 0.97
Train Accuracy : 0.91
Test Accuracy : 0.81
Classification Report for Train:
               precision    recall  f1-score   support

           0       0.86      0.98      0.91      4361
           1       0.94      0.91      0.92      3109
           2       0.91      0.96      0.94      3795
           3       0.96      0.75      0.84      1627
           4       0.96      0.85      0.90      2912

    accuracy                           0.91     15804
   macro avg       0.93      0.89      0.90     15804
weighted avg       0.92      0.91      0.91     15804

Classification Report for Test:
               precision    recall  f1-score   support

           0       0.75      0.94      0.83      1454
           1       0.81      0.75      0.78      1037
           2       0.82      0.89      0.86      1265
           3       0.92      0.61      0.73       542
           4       0.86      0.67      0.75       970

    accuracy                           0.81      5268
   macro avg       0.83      0.77      0.79      5268
weighted avg       0.82      0.81      0.80      5268


table = {'Metric': ['ROC_AUC Score(Train)','ROC_AUC Score(Test)',
                    'Accuracy(Train)','Accuracy(Test)',
                    'Precision(Train)','Precision(Test)',
                    'Recall(Train)','Recall(Test)',
                    'F1-Score(Train)','F1-Score(Test)'
                   ], 
        'Multinomial Naive Bayes': metric1
        }

mnb_metric = pd.DataFrame(table ,columns = ['Metric', 'Multinomial Naive Bayes'] )
log_metric = pd.Series(metric2, name = 'Logistic Regression')
dtc_metric = pd.Series(metric3, name = 'Decision Tree Classifier')
rfc_metric = pd.Series(metric4, name = 'Random Forest Classifier')
grid_mnb_metric = pd.Series(metric5, name = 'Multinomial Naive Bayes with GridSearchCV')
grid_log_metric = pd.Series(metric6, name = 'Logistic Regression with GridSearchCV')
grid_dtc_metric = pd.Series(metric7, name = 'Decision Tree Classifier with GridSearchCV')
grid_rfc_metric = pd.Series(metric8, name = 'Random Forest Classifier with GridSearchCV')

final_metric = pd.concat([mnb_metric,log_metric,dtc_metric,rfc_metric,
                         grid_mnb_metric,grid_log_metric,grid_dtc_metric,grid_rfc_metric], axis = 1)

final_metric


# raw complaint text
test_complaint= 'I tried to make a transaction at a supermarket retail store, using my chase \
debit/atm card, but the transaction was declined. I am still able to withdraw money out of an \
ATM machine using the same debit card. Please resolve this issue.'


# vectorize and tf-idf tranform
test = count_vect.transform([test_complaint])
test_tfidf = tfidf_transformer.transform(test)


# predict
prediction=grid_search_log.predict(test_tfidf)
prediction


topic_mapping[prediction[0]]


## Front End ###

# to run front end head on to the "NLPProject" and run command "python ./manage.py runserver"

	_index	_type	_id	_source.tags	_source.zip_code	_source.complaint_id	_source.issue	_source.date_received	_source.state	_source.consumer_disputed	_source.product	_source.company_response	_source.company	_source.submitted_via	_source.date_sent_to_company	_source.company_public_response	_source.sub_product	_source.timely	_source.complaint_what_happened	_source.sub_issue	_source.consumer_consent_provided
0	complaint-public-v2	complaint	3211475	None	90301	3211475	Attempts to collect debt not owed	2019-04-13T12:00:00-05:00	CA	N/A	Debt collection	Closed with explanation	JPMORGAN CHASE & CO.	Web	2019-04-13T12:00:00-05:00	None	Credit card debt	Yes		Debt is not yours	Consent not provided
1	complaint-public-v2	complaint	3229299	Servicemember	319XX	3229299	Written notification about debt	2019-05-01T12:00:00-05:00	GA	N/A	Debt collection	Closed with explanation	JPMORGAN CHASE & CO.	Web	2019-05-01T12:00:00-05:00	None	Credit card debt	Yes	Good morning my name is XXXX XXXX and I apprec...	Didn't receive enough information to verify debt	Consent provided
2	complaint-public-v2	complaint	3199379	None	77069	3199379	Other features, terms, or problems	2019-04-02T12:00:00-05:00	TX	N/A	Credit card or prepaid card	Closed with explanation	JPMORGAN CHASE & CO.	Web	2019-04-02T12:00:00-05:00	None	General-purpose credit card or charge card	Yes	I upgraded my XXXX XXXX card in XX/XX/2018 and...	Problem with rewards from credit card	Consent provided
3	complaint-public-v2	complaint	2673060	None	48066	2673060	Trouble during payment process	2017-09-13T12:00:00-05:00	MI	N/A	Mortgage	Closed with explanation	JPMORGAN CHASE & CO.	Web	2017-09-14T12:00:00-05:00	None	Conventional home mortgage	Yes		None	Consent not provided
4	complaint-public-v2	complaint	3203545	None	10473	3203545	Fees or interest	2019-04-05T12:00:00-05:00	NY	N/A	Credit card or prepaid card	Closed with explanation	JPMORGAN CHASE & CO.	Referral	2019-04-05T12:00:00-05:00	None	General-purpose credit card or charge card	Yes		Charged too much interest	N/A
5	complaint-public-v2	complaint	3275312	Older American	48227	3275312	Managing an account	2019-06-13T12:00:00-05:00	MI	N/A	Checking or savings account	Closed with monetary relief	JPMORGAN CHASE & CO.	Referral	2019-06-14T12:00:00-05:00	None	Checking account	Yes		Problem using a debit or ATM card	N/A
6	complaint-public-v2	complaint	3238804	None	76262	3238804	Managing an account	2019-05-10T12:00:00-05:00	TX	N/A	Checking or savings account	Closed with monetary relief	JPMORGAN CHASE & CO.	Phone	2019-05-10T12:00:00-05:00	None	Checking account	Yes		Problem using a debit or ATM card	N/A
7	complaint-public-v2	complaint	3249272	None	07753	3249272	Trouble during payment process	2019-05-15T12:00:00-05:00	NJ	N/A	Mortgage	Closed with explanation	JPMORGAN CHASE & CO.	Referral	2019-05-21T12:00:00-05:00	None	Conventional home mortgage	Yes		None	N/A
8	complaint-public-v2	complaint	3351653	None	60621	3351653	Closing an account	2019-08-22T12:00:00-05:00	IL	N/A	Checking or savings account	Closed with explanation	JPMORGAN CHASE & CO.	Referral	2019-08-23T12:00:00-05:00	None	Checking account	Yes		Company closed your account	N/A
9	complaint-public-v2	complaint	3273612	None	99354	3273612	Managing an account	2019-06-11T12:00:00-05:00	WA	N/A	Checking or savings account	Closed with monetary relief	JPMORGAN CHASE & CO.	Referral	2019-06-17T12:00:00-05:00	None	Checking account	Yes		Cashing a check	N/A

	_source.complaint_what_happened	_source.product	_source.sub_product
78308		Checking or savings account	Checking account
78309	On Wednesday, XX/XX/XXXX I called Chas, my XXX...	Credit card or prepaid card	General-purpose credit card or charge card
78310	I am not familiar with XXXX pay and did not un...	Checking or savings account	Checking account
78311	I have had flawless credit for 30 yrs. I've ha...	Credit card or prepaid card	General-purpose credit card or charge card
78312	Roughly 10+ years ago I closed out my accounts...	Payday loan	None

	complaint_text	category	sub_category
0		Debt collection	Credit card debt
1	Good morning my name is XXXX XXXX and I apprec...	Debt collection	Credit card debt
2	I upgraded my XXXX XXXX card in XX/XX/2018 and...	Credit card or prepaid card	General-purpose credit card or charge card
3		Mortgage	Conventional home mortgage
4		Credit card or prepaid card	General-purpose credit card or charge card

	complaint_text	category
0		Debt collection+Credit card debt
1	Good morning my name is XXXX XXXX and I apprec...	Debt collection+Credit card debt
2	I upgraded my XXXX XXXX card in XX/XX/2018 and...	Credit card or prepaid card+General-purpose cr...
3		Mortgage+Conventional home mortgage
4		Credit card or prepaid card+General-purpose cr...

	complaint_text	lemma	category	complaint_POS_removed
1	good morning my name is xxxx xxxx and i apprec...	good morning xxxx xxxx appreciate help stop ch...	Debt collection+Credit card debt	morning help bank service chase debt verificat...
2	i upgraded my xxxx xxxx card in and was told b...	upgrade xxxx xxxx card tell agent upgrade anni...	Credit card or prepaid card+General-purpose cr...	card agent upgrade anniversary date change age...
10	chase card was reported on however fraudulent ...	chase card report fraudulent application submi...	Credit reporting, credit repair services, or o...	chase card report application submit identity ...
11	on while trying to book a xxxx xxxx ticket i...	try book xxxx xxxx ticket come offer apply...	Credit reporting, credit repair services, or o...	book ticket come offer ticket reward card info...
14	my grand son give me check for i deposit it in...	grand son check deposit chase account fund cle...	Checking or savings account+Checking account	son deposit chase account fund bank account pa...

	Topic0	Topic1	Topic2	Topic3	Topic4	dominant_topic
Doc0	0.03	0.00	0.02	0.02	0.01	0
Doc1	0.01	0.02	0.00	0.00	0.01	1
Doc2	0.00	0.03	0.00	0.04	0.00	3
Doc3	0.02	0.03	0.01	0.05	0.01	3
Doc4	0.07	0.00	0.01	0.00	0.00	0

	complaint_text	Topic
1	good morning my name is xxxx xxxx and i appreciate it if you could help me put a stop to chase bank cardmember services \nin i wrote to chase asking for debt verification and what they sent me a statement which is not acceptable i am asking the bank to validate the debt instead i been receiving mail every month from them attempting to collect a debt \ni have a right to know this information as a consumer \n\nchase account xxxx xxxx xxxx xxxx thanks in advance for your help	Bank Account services
2	i upgraded my xxxx xxxx card in and was told by the agent who did the upgrade my anniversary date would not change it turned the agent was giving me the wrong information in order to upgrade the account xxxx changed my anniversary date from xxxxxxxx to xxxxxxxx without my consent xxxx has the recording of the agent who was misled me	Credit card or prepaid card
10	chase card was reported on however fraudulent application have been submitted my identity without my consent to fraudulently obtain services do not extend credit without verifying the identity of the applicant	Theft/Dispute Reporting
11	on while trying to book a xxxx xxxx ticket i came across an offer for to be applied towards the ticket if i applied for a rewards card i put in my information for the offer and within less than a minute was notified via the screen that a decision could not be made i immediately contacted xxxx and was referred to chase bank i then immediately contacted chase bank within no more than of getting the notification on the screen and i was told by the chase representative i spoke with that my application was denied but she could not state why i asked for more information about the xxxx offer and she explained that even if i had been approved the credit offer only gets applied after the first account statement and could not be used to purchase the ticket i then explicitly told her i was glad i got denied and i was absolutely no longer interested in the account i asked that the application be withdrawn and the representative obliged this all happened no later than after putting in the application on notwithstanding my explicit request not to proceed with the application and contrary to what i was told by the chase representative chase did in fact go ahead to open a credit account in my name on this is now being reported in my credit report and chase has refused to correct this information on my credit report even though they went ahead to process an application which i did not consent to and out of their error	Theft/Dispute Reporting
14	my grand son give me check for i deposit it into my chase account after fund clear my chase bank closed my account never paid me my money they said they need to speek with my grand son check was clear money was taking by my chase bank refuse to pay me my money my grand son called chase times they told him i should call not him to verify the check owner he is out the country most the time date happen check number xxxx claim number is xxxx with chase	Bank Account services

	Metric	Multinomial Naive Bayes	Logistic Regression	Decision Tree Classifier	Random Forest Classifier	Multinomial Naive Bayes with GridSearchCV	Logistic Regression with GridSearchCV	Decision Tree Classifier with GridSearchCV	Random Forest Classifier with GridSearchCV
0	ROC_AUC Score(Train)	0.95	1.00	1.00	1.00	0.99	1.00	0.97	0.99
1	ROC_AUC Score(Test)	0.93	0.99	0.85	0.98	0.94	0.99	0.94	0.97
2	Accuracy(Train)	0.68	0.94	1.00	1.00	0.89	0.94	0.82	0.91
3	Accuracy(Test)	0.64	0.90	0.76	0.83	0.76	0.93	0.78	0.81
4	Precision(Train)	0.76	0.94	1.00	1.00	0.89	0.94	0.82	0.92
5	Precision(Test)	0.72	0.90	0.76	0.84	0.76	0.93	0.78	0.82
6	Recall(Train)	0.68	0.94	1.00	1.00	0.89	0.94	0.82	0.91
7	Recall(Test)	0.64	0.90	0.76	0.83	0.76	0.93	0.78	0.81
8	F1-Score(Train)	0.65	0.94	1.00	1.00	0.89	0.94	0.82	0.91
9	F1-Score(Test)	0.61	0.90	0.76	0.83	0.75	0.93	0.78	0.80

Introduction¶

Understanding Problem Statement¶

Business goal¶

Pipelines performed:¶

Importing the necessary libraries¶

Data Loading¶

Data preparation¶

Text Preprocessing¶

Prepare the text for topic modeling¶

Exploratory data analysis (EDA)¶

Exploratory data analysis to get familiar with the data.¶

Compaints by Products' Categories¶

Complain Resolution Status¶

Distribution of Compalints Over Time¶

Highest Number of Complaints by States¶

Comparison of Timely and Untimely Complaints¶

Finding the top 40 words by frequency among all the articles after processing the text.¶

WordCloud Before Cleaning¶

WordCloud After Cleaning¶

Find the top unigrams,bigrams and trigrams by frequency among all the complaints after processing the text.¶

The personal details of customer has been masked in the dataset with xxxx. Removing the masked text as this will be of no use for analysis¶

Feature Extraction¶

Convert the raw texts to a matrix of TF-IDF features¶

Creating a document term matrix using fit_transform¶

Topic Modelling¶

Topic Modelling using NMF¶

Manual Topic Modeling¶

After evaluating the mapping, if the topics assigned are correct then assign these names to the relevant topic:¶

Lets check if there is any class imbalance¶

Model building using supervised learning, Model training & evaluation¶

Supervised model to predict any new complaints to the relevant Topics.¶

Reverse topic names mapping for supervised learning¶

Apply the supervised models on the training data created. In this process, you have to do the following:¶

X - Y split¶

Train Test Split¶

We are trying the following models -¶

Logistic regression, Decision Tree & Random Forest¶

Custom Classification Report¶

Stratified KFold Cross Validation¶

Custom GridSearchCV¶

Multinomial Naive Bayes Classification¶

Logistic Regression Classification¶

Decision Tree Classification¶

Random Forest Classification¶

Hyper parameter tuning with GridSearchCV¶

Multinomial Naive Bayes with GridSearchCV¶

Logistic Regression with GridSearchCV¶

Decision Tree Classification with GridSearchCV¶¶

Random Forest Classifier with GridSearchCV¶

Model inference¶

Consolidated view of all the models and their metrics¶

Complaint Prediction using the best model above¶

The best model looks to be Logistic Regression with GridSearchCV¶